#include"../Include/WebPageLib.h"
#include"../Include/Configuration.h"
#include"../Include/SplitToolCppJieba.h"

using std::cout;
using std::endl;
using std::ofstream;
using std::string;
using std::vector;

WebPageLib::WebPageLib(DirScanner& dirScanner,RssReader& rssReader)
:_dirScanner(dirScanner),_rssReader(rssReader){}

WebPageLib::~WebPageLib(){}

void WebPageLib::createWebPageLib()
{
    _dirScanner(Configuration::getInstance().getWebXmlPath());
    std::vector<std::string>& files = _dirScanner.files();
    //日志点 扫描xml目录
    
    _rssReader.parseRss(files);
    _rssReader.deDuplicateBySimHash();
    _files = _rssReader.formatToString();
    //日志点 存放格式化后的xml文件

    //日志点 完成网页库的初始化
}

void WebPageLib::storeWebPagesFile()
{

    ofstream ofs(Configuration::getInstance().getFormatedWebPagePath().c_str());
    //日志点 读取失败

    for(auto& item : _files)
    {
        ofs<<item;
    }

    ofs.close();
}

void WebPageLib::storeOffsetFile()
{
    ofstream ofs(Configuration::getInstance().getOffsetFilePath().c_str());
    //日志点 读取失败

    int lentmp = 0;
    for(int index = 0;index<_files.size();++index)
    {
        string str = "" + to_string(index + 1) 
                        + " "  + to_string(lentmp) 
                        + " "  + to_string(_file[index].size());
        ofs << str << endl;
        lentmp += _files[index].size(); 
    }
    ofs.close();
}

void WebPageLib::storeInvertIndexFile()
{
   ofstream ofs(Configuration::getInstance().getInvertIndexPath().c_str());
  //日志点 读取失败

   for(auto& outside:_invert)
   {
       string str = "" + outside.first;
       for(auto& inside : outside.second)
       {
           str += (" " + to_string(inside.first + 1) +
                   " " + to_string(inside.second));
       }
       ofs << str << std::endl;
   }

   ofs.close();
}

void WebPageLib::setInvertIndex()
{
    SplitToollCppJieba jieba(Configuration::getInstance());

    auto stopWords = Configuration::getInstance().getChineseStopWordList();
    unordered_map<string,unordered_map<int,int>> tf_map;

    for(int i = 0;i<_files.size();++i)
    {
      auto words = jieba.cut(_files[i]);
      for(auto& word : words)
      {
          if(stopWords.find(word)==stopWords.end())
          {
              ++tf_map[word][i];
          }
      }
    }

    unordered_map<string,unordered_map<int,double>> invert;
    int N = _file.size();
    for(auto& outside : tf_map)
    {
        int df = outside.second.size();
        for(auto& inside : outside.second)
        {
            invert[outside.first][inside.first] = 
                (inside.second * log2(N/(df + 1.0)));
        }
    }

    for(auto& outside : id_word)
    {
        double ww = 0.0;
        for(auto& word : outside.second)
        {
            ww += pow(invert[word][outside.first],2);
        }

        for(auto& word : outside.second )
        {
            _invert[word].push_back(make_pair(
            outside.first,invert[word][outside.first]/sqrt(ww)));
            
        }
    }
}

void WebPageLib::store()
{
    storeWebPagesFile();
    storeOffsetFile();
    storeInvertIndexFile();
}
